package filter;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

public class SignBiasMF {

public static class Data {
	public int uid;
	public short iid;
	public byte rat;
	
	public Data(){
		uid = 0;
		iid = 0;
		rat = 0;
	}
	
	public Data(int uid, short iid, byte rat){
		this.uid = uid;
		this.iid = iid;
		this.rat = rat;
	}
}
	
	public static double RMSE(List<Data> dataList, double mean, double[] ubias, double[] ibias, double[] ufactor, double[] ifactor, int nfactor) {
		double sumsqrerr = 0.0;
		int nrat = dataList.size();
		
		for(int i=0; i<nrat; i++) {
			int uid = dataList.get(i).uid;
			short iid = dataList.get(i).iid;
			byte rat = dataList.get(i).rat;

			double pred = 0;
			pred += mean;
			pred += ubias[uid-1];
			pred += ibias[iid-1];
			for(int j=0; j<nfactor; j++){
				pred += ufactor[(uid-1)*nfactor+j] * ifactor[(iid-1)*nfactor+j];
			}
			double err = rat - pred;
			sumsqrerr += err*err;
		}
		double rmse = sumsqrerr / nrat;
//		System.out.println(rmse);
		
		return rmse;
	}
	
	public static double RMSE(List<Data> posList, List<Data> negList, double posmean, double negmean, double[] uposbias, double[] iposbias, double[] uposfactor, double[] iposfactor, double[] unegbias, double[] inegbias, double[] unegfactor, double[] inegfactor, int nfactor) {
		double sumsqrerr = 0.0;
		int nrat = posList.size() + negList.size();
		
		for(Data data : posList) {
			int uid = data.uid;
			short iid = data.iid;
			byte rat = data.rat;

			double pred = 0;
			pred += posmean;
			pred += uposbias[uid-1];
			pred += iposbias[iid-1];
			for(int j=0; j<nfactor; j++){
				pred += uposfactor[(uid-1)*nfactor+j] * iposfactor[(iid-1)*nfactor+j];
			}
			
			double err = rat - pred;
			sumsqrerr += err*err;
		}
		
		for(Data data : negList) {
			int uid = data.uid;
			short iid = data.iid;
			byte rat = data.rat;

			double pred = 0;
			pred += negmean;
			pred += unegbias[uid-1];
			pred += inegbias[iid-1];
			for(int j=0; j<nfactor; j++){
				pred += unegfactor[(uid-1)*nfactor+j] * inegfactor[(iid-1)*nfactor+j];
			}
			
			double err = rat - pred;
			sumsqrerr += err*err;
		}
		
		double rmse = sumsqrerr / nrat;
//		System.out.println(rmse);
		
		return rmse;
	}
	
	public static void main(String[] args) throws IOException {
		System.out.println("Program started...");
		int nrat = 3298163;
		int nuser = 93705;
		int nitem = 3561;
		int maxuid = 95526;
		String dir = "D:/Netflix Dataset/download/training_set/";
		
		int[] usumrat = new int[maxuid];
		int[] isumrat = new int[nitem];
		
		for(int i=0; i<maxuid; i++) {
			usumrat[i] = 0;
		}		
		for(int i=0; i<nitem; i++) {
			isumrat[i] = 0;
		}
		
		int[] unrat = new int[maxuid];
		int[] inrat = new int[nitem];
		
		for(int i=0; i<maxuid; i++) {
			unrat[i] = 0;
		}
		for(int i=0; i<nitem; i++) {
			inrat[i] = 0;
		}
		
		Set<Integer> uset = new HashSet<Integer>();
		Set<Short> iset = new HashSet<Short>();
		
		//read training set		
		List<Data> train = new ArrayList<Data>();
		BufferedReader tr = new BufferedReader(new FileReader(dir + "smalltrain"));
		String line=null;
		double sumrat = 0.0;
		
		while((line = tr.readLine()) != null) {
			String[] fields = line.split(",");
			int uid = Integer.parseInt(fields[0]);
			short iid = Short.parseShort(fields[1]);
			byte rat = Byte.parseByte(fields[2]);	
			
			train.add(new Data(uid,iid,rat));
			
			uset.add(uid);
			iset.add(iid);
			
			usumrat[uid-1] += rat;
			isumrat[iid-1] += rat;
			
			unrat[uid-1]++;
			inrat[iid-1]++;
			
			sumrat += rat;
		}
		
		tr.close();
		
		double avgrat = sumrat/nrat;
		System.out.println("Training mean rating: " + avgrat);
		
		double[] uavgrat = new double[maxuid];
		double[] iavgrat = new double[nitem];
		
		for(int i=0; i<maxuid; i++){
			uavgrat[i] = usumrat[i] / (double) unrat[i];
		}
		for(int i=0; i<nitem; i++){
			iavgrat[i] = isumrat[i] / (double) inrat[i];
		}
		
		//read validation set		
		List<Data> valid = new ArrayList<Data>();
		BufferedReader va = new BufferedReader(new FileReader(dir + "smallvalidate"));
		double vasumrat = 0.0;
		int vanrat = 0;
		
		while((line = va.readLine()) != null) {
			String[] fields = line.split(",");
			int uid = Integer.parseInt(fields[0]);
			short iid = Short.parseShort(fields[1]);
			byte rat = Byte.parseByte(fields[2]);	
			
			valid.add(new Data(uid,iid,rat));
			
			vasumrat += rat;
			vanrat++;
		}
		double vaavgrat = vasumrat/vanrat;
		System.out.println("Validation mean rating: " + vaavgrat);
		
		va.close();
		
		//model params
		byte k = 20;
		double gamma = (double) 0.02;		
		double lambda =(double) 0.02;
		
		System.out.println("Dimensionality k = " + k);
		System.out.println("Gamma = " + gamma);
		System.out.println("Lambda = " + lambda);
		
		double[] ubias = new double[maxuid];
		double[] ibias = new double[nitem];
		
		for(int i=0; i<maxuid; i++) {
			ubias[i] = 0.0;
		}
		for(int i=0; i<nitem; i++) {
			ibias[i] = 0.0;
		}
		
		double[] ufactor = new double[maxuid * k];
		double[] ifactor = new double[nitem * k];
		
		//set factor randomly
		for(int i=0; i<nuser*k; i++) {
			ufactor[i] = (Math.random()-0.5)/(double)k;
		}
		for(int i=0; i<nitem*k; i++) {
			ifactor[i] = (Math.random()-0.5)/(double)k;
		}
		
		//initial error
		System.out.println("Initial training RMSE: " + RMSE(train, avgrat, ubias, ibias, ufactor, ifactor, k));
		
		//learn
		for (int it=1;it<=100;it++){
			Collections.shuffle(train);
			double formerrmse = RMSE(train, avgrat, ubias, ibias, ufactor, ifactor, k);
			for(Data data : train) {
				int uid = data.uid;
				short iid = data.iid;
				byte rat = data.rat;

				double pred = 0.0;
				pred += avgrat;
				pred += ubias[uid-1];
				pred += ibias[iid-1];
				for(int j=0; j<k; j++) {
					pred += ufactor[(uid-1)*k+j]*ifactor[(iid-1)*k+j];
				}
				double err = rat - pred;
				
				ubias[uid-1] =  ubias[uid-1] + gamma*(err-lambda*ubias[uid-1]);
				ibias[iid-1] =  ibias[iid-1] + gamma*(err-lambda*ibias[iid-1]);
				
				for(int j=0; j<k; j++) {
					double newufactor = ufactor[(uid-1)*k+j] + gamma*(err*ifactor[(iid-1)*k+j] - lambda*ufactor[(uid-1)*k+j]);
					double newifactor = ifactor[(iid-1)*k+j] + gamma*(err*ufactor[(uid-1)*k+j] - lambda*ifactor[(iid-1)*k+j]);
					ufactor[(uid-1)*k+j] = newufactor;
					ifactor[(iid-1)*k+j] = newifactor;
				}
			}
			double latterrmse = RMSE(train, avgrat, ubias, ibias, ufactor, ifactor, k);
			if (formerrmse - latterrmse < 0.0001){
				System.out.println("Converged at step " + it);
				break;
			}
		}
		
		//error after learning
		System.out.println("Training RMSE after learning: " + RMSE(train, avgrat, ubias, ibias, ufactor, ifactor, k));		
		System.out.println("Validation RMSE from normal MF: " + RMSE(valid, avgrat, ubias, ibias, ufactor, ifactor, k));
		
		int tp = 0;
		int fp = 0;
		int tn = 0;
		int fn = 0;
		
		List<Data> train2 = new ArrayList<Data>();
		
		for(int i=0; i<train.size(); i++) {
			Data data = train.get(i);
			int uid = data.uid;
			short iid = data.iid;
			byte rat = data.rat;
			
			double pred = avgrat + ubias[uid-1] + ibias[iid-1];
			
			double dot = 0.0;			
			for(int j=0; j<k; j++){
				dot += ufactor[(uid-1)*k+j] * ifactor[(iid-1)*k+j];
			}
			
			if( rat - pred > 0 && dot > 0 ){
				tp++;
				train2.add(data);
			}
			if( rat - pred < 0 && dot > 0 ){
				fp++;
				uset.remove(uid);
				iset.remove(iid);
			}
			if( rat - pred > 0 && dot < 0 ){
				fn++;
				uset.remove(uid);
				iset.remove(iid);
			}
			if( rat - pred < 0 && dot < 0){
				tn++;
				train2.add(data);
			}			
		}
		
		System.out.println("New training set size: "+train2.size());
		
		//initial error
		System.out.println("Initial training RMSE: " + RMSE(train2, avgrat, ubias, ibias, ufactor, ifactor, k));
		
		//learn
		for (int it=1;it<=100;it++){
			Collections.shuffle(train2);
			double formerrmse = RMSE(train2, avgrat, ubias, ibias, ufactor, ifactor, k);
			for(Data data : train2) {
				int uid = data.uid;
				short iid = data.iid;
				byte rat = data.rat;

				double pred = 0.0;
				pred += avgrat;
				pred += ubias[uid-1];
				pred += ibias[iid-1];
				for(int j=0; j<k; j++) {
					pred += ufactor[(uid-1)*k+j]*ifactor[(iid-1)*k+j];
				}
				double err = rat - pred;
				
				ubias[uid-1] =  ubias[uid-1] + gamma*(err-lambda*ubias[uid-1]);
				ibias[iid-1] =  ibias[iid-1] + gamma*(err-lambda*ibias[iid-1]);
				
				for(int j=0; j<k; j++) {
					double newufactor = ufactor[(uid-1)*k+j] + gamma*(err*ifactor[(iid-1)*k+j] - lambda*ufactor[(uid-1)*k+j]);
					double newifactor = ifactor[(iid-1)*k+j] + gamma*(err*ufactor[(uid-1)*k+j] - lambda*ifactor[(iid-1)*k+j]);
					ufactor[(uid-1)*k+j] = newufactor;
					ifactor[(iid-1)*k+j] = newifactor;
				}
			}
			double latterrmse = RMSE(train2, avgrat, ubias, ibias, ufactor, ifactor, k);
			if (formerrmse - latterrmse < 0.0001){
				System.out.println("Converged at step " + it);
				break;
			}
		}
		
		//error after learning
		System.out.println("Training RMSE after learning: " + RMSE(train2, avgrat, ubias, ibias, ufactor, ifactor, k));		
		System.out.println("Validation RMSE from normal MF: " + RMSE(valid, avgrat, ubias, ibias, ufactor, ifactor, k));
		
		/*System.out.println("No. of users: "+uset.size());
		System.out.println("No. of items: "+iset.size());
		
		System.out.println("Training Accuracy: "+((tp+tn)/(double)(tp+fp+tn+fn)));
		System.out.println("Training Precision/Recall for positive class: "+(tp/(double)(tp+fp))+"/"+(tp/(double)(tp+fn)));
		System.out.println("Training Precision/Recall for negative class: "+(tn/(double)(tn+fn))+"/"+(tn/(double)(tn+fp)));
		
		 tp = 0;
		 fp = 0;
		 tn = 0;
		 fn = 0;
		
		for(Data data : valid) {
			int uid = data.uid;
			short iid = data.iid;
			byte rat = data.rat;
			
			double pred = avgrat + ubias[uid-1] + ibias[iid-1];
			
			double dot = 0.0;			
			for(int i=0; i<k; i++){
				dot += ufactor[(uid-1)*k+i] * ifactor[(iid-1)*k+i];
			}
			
			if(uset.contains(uid) && iset.contains(iid)) {
				
				if( rat - pred > 0 && dot > 0 ){
					tp++;
				}
				if( rat - pred < 0 && dot > 0 ){
					fp++;
				}
				if( rat - pred > 0 && dot < 0 ){
					fn++;
				}
				if( rat - pred < 0 && dot < 0){
					tn++;
				}			
			}
		}
		
		System.out.println("Validation Accuracy: "+((tp+tn)/(double)(tp+fp+tn+fn)));
		System.out.println("Validation Precision/Recall for positive class: "+(tp/(double)(tp+fp))+"/"+(tp/(double)(tp+fn)));
		System.out.println("Validation Precision/Recall for negative class: "+(tn/(double)(tn+fn))+"/"+(tn/(double)(tn+fp)));*/
	}

}
